{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "613a232a-fdc1-44e8-b084-34d7d66dff3f",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'PlotSettings' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[1], line 11\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mxgboost\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m XGBClassifier\n\u001b[0;32m      7\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mPyQt5\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m QtCore, QtWidgets, uic\n\u001b[1;32m---> 11\u001b[0m \u001b[38;5;28;43;01mclass\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;21;43;01mMainApp\u001b[39;49;00m\u001b[43m(\u001b[49m\u001b[43mQtWidgets\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mQMainWindow\u001b[49m\u001b[43m)\u001b[49m\u001b[43m:\u001b[49m\n\u001b[0;32m     12\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43;01mdef\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m:\u001b[49m\n\u001b[0;32m     13\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mMainApp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[1;32mIn[1], line 181\u001b[0m, in \u001b[0;36mMainApp\u001b[1;34m()\u001b[0m\n\u001b[0;32m    171\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mui\u001b[38;5;241m.\u001b[39msetupUi(\n\u001b[0;32m    172\u001b[0m         dialog,\n\u001b[0;32m    173\u001b[0m         \u001b[38;5;28mlist\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset\u001b[38;5;241m.\u001b[39mcolumns),\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    177\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mretrieve_data_from_child,\n\u001b[0;32m    178\u001b[0m     )\n\u001b[0;32m    179\u001b[0m     dialog\u001b[38;5;241m.\u001b[39mshow()\n\u001b[1;32m--> 181\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mretrieve_data_from_child\u001b[39m(\u001b[38;5;28mself\u001b[39m, settings: \u001b[43mPlotSettings\u001b[49m):\n\u001b[0;32m    182\u001b[0m     \u001b[38;5;66;03m# self.createPlots(\u001b[39;00m\n\u001b[0;32m    183\u001b[0m     \u001b[38;5;66;03m#     settings.column,\u001b[39;00m\n\u001b[0;32m    184\u001b[0m     \u001b[38;5;66;03m#     settings.min_value,\u001b[39;00m\n\u001b[0;32m    185\u001b[0m     \u001b[38;5;66;03m#     settings.max_value,\u001b[39;00m\n\u001b[0;32m    186\u001b[0m     \u001b[38;5;66;03m#     settings.category_column,\u001b[39;00m\n\u001b[0;32m    187\u001b[0m     \u001b[38;5;66;03m# )\u001b[39;00m\n\u001b[0;32m    188\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_plots_in_progress \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m    190\u001b[0m     plot_creator \u001b[38;5;241m=\u001b[39m PlotCreator(\n\u001b[0;32m    191\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel,\n\u001b[0;32m    192\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    195\u001b[0m         settings\u001b[38;5;241m.\u001b[39mmax_value,\n\u001b[0;32m    196\u001b[0m     )\n",
      "\u001b[1;31mNameError\u001b[0m: name 'PlotSettings' is not defined"
     ]
    }
   ],
   "source": [
    "import tempfile\n",
    "from typing import Any\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from pandas import DataFrame\n",
    "from xgboost import XGBClassifier\n",
    "from PyQt5 import QtCore, QtWidgets, uic\n",
    "\n",
    "\n",
    "\n",
    "class MainApp(QtWidgets.QMainWindow):\n",
    "    def __init__(self, parent=None):\n",
    "        super(MainApp, self).__init__()\n",
    "\n",
    "        self.__qt_helper = QtHelper()\n",
    "        self.__pickle_service = PickleService()\n",
    "\n",
    "        uic.loadUi(\"../ui/main.ui\", self)\n",
    "\n",
    "        self.open_file_action.setStatusTip(\"Открыть файл проекта\")\n",
    "        self.open_file_action.triggered.connect(self.open_saved_file)\n",
    "\n",
    "        self.save_file_action.setStatusTip(\"Сохранить файл модели и выборки\")\n",
    "        self.save_file_action.triggered.connect(self.save_file)\n",
    "\n",
    "        self.load_model_action.setStatusTip(\"Открыть файл модели\")\n",
    "        self.load_model_action.triggered.connect(self.open_model)\n",
    "\n",
    "        self.load_dataset_action.setStatusTip(\"Открыть файл выборки\")\n",
    "        self.load_dataset_action.triggered.connect(self.open_dataset)\n",
    "\n",
    "        self.save_plot_action.triggered.connect(self.save_plots)\n",
    "\n",
    "        self.push_button_plot.clicked.connect(self.make_plots)\n",
    "\n",
    "        self.plots_combo_box.currentIndexChanged.connect(self.switch_plots)\n",
    "\n",
    "        self.statusBar().showMessage(\"Ожидается загрузка модели и датасета\")\n",
    "\n",
    "        self.is_model_loaded = False\n",
    "        self.is_dataset_loaded = False\n",
    "\n",
    "        self.__model = None\n",
    "        self.__dataset = None\n",
    "\n",
    "        self.__paths: dict[str, list[str]] = {}\n",
    "\n",
    "        self.temp_dir = tempfile.TemporaryDirectory()\n",
    "\n",
    "        self.setWindowState(QtCore.Qt.WindowMaximized)\n",
    "        self.debug_start()\n",
    "        self.show()\n",
    "\n",
    "    @property\n",
    "    def dataset(self) -> DataFrame:\n",
    "        return self.__dataset\n",
    "\n",
    "    @dataset.setter\n",
    "    def dataset(self, dataset) -> None:\n",
    "        self.is_dataset_loaded = True\n",
    "        self.__dataset = dataset\n",
    "\n",
    "        self.create_columns()\n",
    "        self.create_rows()\n",
    "        self.show_data_status()\n",
    "\n",
    "    @property\n",
    "    def model(self) -> XGBClassifier:\n",
    "        return self.__model\n",
    "\n",
    "    @model.setter\n",
    "    def model(self, model) -> None:\n",
    "        self.is_model_loaded = True\n",
    "        self.__model = model\n",
    "        self.show_data_status()\n",
    "\n",
    "    def closeEvent(self, event):\n",
    "        self.temp_dir.cleanup()\n",
    "\n",
    "    def debug_start(self):\n",
    "        self.open_saved_file(debug_file_path)\n",
    "\n",
    "    def show_data_status(self) -> None:\n",
    "        if self.is_dataset_loaded and self.is_model_loaded:\n",
    "            self.statusBar().showMessage(dataset_and_model_loaded_message)\n",
    "        elif self.is_dataset_loaded:\n",
    "            self.statusBar().showMessage(only_dataset_loaded_message)\n",
    "        elif self.is_model_loaded:\n",
    "            self.statusBar().showMessage(only_model_loaded_message)\n",
    "        else:\n",
    "            self.statusBar().showMessage(data_is_awaited_message)\n",
    "\n",
    "    def open_saved_file(self, path: str = \"\"):\n",
    "        if not path:\n",
    "            file_name = self.__qt_helper.get_path_to_open_file(self)\n",
    "        else:\n",
    "            file_name = path\n",
    "\n",
    "        if not file_name:\n",
    "            return\n",
    "\n",
    "        mono_object = self.__pickle_service.get_dataset_and_model(file_name)\n",
    "\n",
    "        self.model = mono_object.model\n",
    "        self.dataset = mono_object.dataset\n",
    "\n",
    "    def save_file(self):\n",
    "        if not all([self.is_dataset_loaded, self.is_model_loaded]):\n",
    "            self.show_data_status()\n",
    "            return\n",
    "\n",
    "        file_name = self.__qt_helper.get_path_to_save_file(self)\n",
    "\n",
    "        if not file_name:\n",
    "            return\n",
    "\n",
    "        mono = DatasetModelMonoObject(dataset=self.__dataset, model=self.model)\n",
    "        self.__pickle_service.save_dataset_and_model(mono, file_name)\n",
    "\n",
    "        self.statusBar().showMessage(save_complete_message)\n",
    "\n",
    "    def open_dataset(self):\n",
    "        path = self.__qt_helper.get_path_to_open_file(self)\n",
    "\n",
    "        if not path:\n",
    "            return None\n",
    "\n",
    "        self.dataset = self.__pickle_service.get_dataset(path)\n",
    "\n",
    "    def create_columns(self):\n",
    "        layout = DatasetRendered(self.dataset).get_rendered_info_plots_layout()\n",
    "        self.tabCollumns.setLayout(layout)\n",
    "\n",
    "    def create_rows(self):\n",
    "        layout = DatasetRendered(self.dataset).get_rendered_data_layout()\n",
    "        self.tabData.setLayout(layout)\n",
    "\n",
    "    def open_model(self):\n",
    "        file_name = self.__qt_helper.get_path_to_open_file(self)\n",
    "\n",
    "        if not file_name:\n",
    "            return\n",
    "\n",
    "        self.model = self.__pickle_service.get_model(file_name)\n",
    "\n",
    "    def save_plots(self):\n",
    "        if self.plots_combo_box.count() == 0:\n",
    "            self.statusBar().showMessage(no_plots_message)\n",
    "            return\n",
    "\n",
    "        path = self.__qt_helper.get_existing_dir_path(self)\n",
    "\n",
    "        if not path:\n",
    "            return\n",
    "\n",
    "        CreatedPlotsSaver.save_plots(self.temp_dir.name, path)\n",
    "\n",
    "    def switch_plots(self):\n",
    "        if not self.is_plots_in_progress:\n",
    "            text = self.plots_combo_box.currentText()\n",
    "            self.show_plots(text)\n",
    "\n",
    "    def make_plots(self):\n",
    "        if not self.is_dataset_loaded and self.is_model_loaded:\n",
    "            self.show_data_status()\n",
    "\n",
    "        dialog = QtWidgets.QDialog()\n",
    "        self.ui = PlotDataDialog()\n",
    "        min_max = getMinMax(self.dataset)\n",
    "        category_columns = find_category_columns(self.dataset)\n",
    "        self.ui.setupUi(\n",
    "            dialog,\n",
    "            list(self.dataset.columns),\n",
    "            category_columns,\n",
    "            min_max,\n",
    "            self,\n",
    "            self.retrieve_data_from_child,\n",
    "        )\n",
    "        dialog.show()\n",
    "\n",
    "    def retrieve_data_from_child(self, settings: PlotSettings):\n",
    "        # self.createPlots(\n",
    "        #     settings.column,\n",
    "        #     settings.min_value,\n",
    "        #     settings.max_value,\n",
    "        #     settings.category_column,\n",
    "        # )\n",
    "        self.is_plots_in_progress = True\n",
    "\n",
    "        plot_creator = PlotCreator(\n",
    "            self.model,\n",
    "            self.dataset,\n",
    "            settings.column,\n",
    "            settings.min_value,\n",
    "            settings.max_value,\n",
    "        )\n",
    "\n",
    "        if settings.category_column:\n",
    "            full_paths = plot_creator.plot_category_plots(settings.category_column)\n",
    "            self.__save_category_paths(full_paths, settings)\n",
    "        else:\n",
    "            paths = plot_creator.plot_regular_plots()\n",
    "            self.__save_regular_paths(paths, settings)\n",
    "\n",
    "        self.is_plots_in_progress = False\n",
    "\n",
    "        self.__update_plot_names_in_combo_box()\n",
    "        self.switch_plots()\n",
    "\n",
    "    def createPlots(self, colName, minVal, maxVal, categorcal_col=\"\"):\n",
    "        categorical = bool(categorcal_col)\n",
    "\n",
    "        cond1 = self.dataset[colName] >= minVal\n",
    "        cond2 = self.dataset[colName] <= maxVal\n",
    "\n",
    "        cur_data = self.dataset[cond1 & cond2].reset_index(drop=True)\n",
    "\n",
    "        h = 6\n",
    "        w = 12\n",
    "        plt.figure(1, figsize=(1000, 1000), dpi=1)\n",
    "\n",
    "        temp = self.temp_dir\n",
    "\n",
    "        self.is_plots_in_progress = True\n",
    "\n",
    "        if categorical:\n",
    "            categories = sorted(list(self.dataset[categorcal_col].unique()))\n",
    "            categories_items = [f\"{colName}:{categorcal_col}:{x}\" for x in categories]\n",
    "            self.plots_combo_box.addItems(categories_items)\n",
    "            self.plots_combo_box.setEnabled(True)\n",
    "            self.plots_combo_box.setCurrentIndex(\n",
    "                self.plots_combo_box.count() - len(categories)\n",
    "            )\n",
    "\n",
    "            for i in range(len(categories)):\n",
    "                plt.clf()\n",
    "\n",
    "                cond3 = self.dataset[categorcal_col] == categories[i]\n",
    "\n",
    "                cur_data = self.dataset[cond1 & cond2 & cond3].reset_index(drop=True)\n",
    "\n",
    "                name = categories_items[i].replace(\":\", \"\")\n",
    "\n",
    "                plot_lt = plot_top5_centered_importance(\n",
    "                    self.model, cur_data, colName, True\n",
    "                )\n",
    "                fig_lt = plot_lt.get_figure()\n",
    "                fig_lt.set_size_inches(w, h)\n",
    "                fig_lt.savefig(temp.name + f\"\\\\img1{name}.svg\", bbox_inches=\"tight\")\n",
    "                fig_lt.savefig(\n",
    "                    temp.name + f\"\\\\img1{name}.png\", bbox_inches=\"tight\", format=\"png\"\n",
    "                )\n",
    "                plt.clf()\n",
    "                #\n",
    "                if colName == categorcal_col:\n",
    "                    plot_lb = plot_ice_plot(\n",
    "                        self.model,\n",
    "                        self.dataset[cond1 & cond2].reset_index(drop=True),\n",
    "                        colName,\n",
    "                        True,\n",
    "                    )\n",
    "                else:\n",
    "                    plot_lb = plot_ice_plot(self.model, cur_data, colName, True)\n",
    "                fig_lb = plot_lb.get_figure()\n",
    "                fig_lb.set_size_inches(w, h)\n",
    "                fig_lb.savefig(temp.name + f\"\\\\img2{name}.svg\", bbox_inches=\"tight\")\n",
    "                fig_lb.savefig(\n",
    "                    temp.name + f\"\\\\img2{name}.png\", bbox_inches=\"tight\", format=\"png\"\n",
    "                )\n",
    "                plt.clf()\n",
    "\n",
    "                plot_rt = plot_top5_centered_importance(self.model, cur_data, colName)\n",
    "                fig_rt = plot_rt.get_figure().figure\n",
    "                fig_rt.set_size_inches(w, h)\n",
    "                fig_rt.savefig(temp.name + f\"\\\\img0{name}.svg\", bbox_inches=\"tight\")\n",
    "                fig_rt.savefig(\n",
    "                    temp.name + f\"\\\\img0{name}.png\", bbox_inches=\"tight\", format=\"png\"\n",
    "                )\n",
    "                plt.clf()\n",
    "                #\n",
    "                if colName == categorcal_col:\n",
    "                    plot_rb = plot_ice_plot(\n",
    "                        self.model,\n",
    "                        self.dataset[cond1 & cond2].reset_index(drop=True),\n",
    "                        colName,\n",
    "                    )\n",
    "                else:\n",
    "                    plot_rb = plot_ice_plot(self.model, cur_data, colName)\n",
    "                fig_rb = plot_rb.get_figure().figure\n",
    "                fig_rb.set_size_inches(w, h)\n",
    "                fig_rb.savefig(temp.name + f\"\\\\img3{name}.svg\", bbox_inches=\"tight\")\n",
    "                fig_rb.savefig(\n",
    "                    temp.name + f\"\\\\img3{name}.png\", bbox_inches=\"tight\", format=\"png\"\n",
    "                )\n",
    "                plt.clf()\n",
    "\n",
    "            self.show_grath(\n",
    "                self.plots_combo_box.itemText(\n",
    "                    self.plots_combo_box.count() - len(categories)\n",
    "                ).replace(\":\", \"\")\n",
    "            )\n",
    "\n",
    "        else:\n",
    "            self.plots_combo_box.addItems([colName])\n",
    "            self.plots_combo_box.setEnabled(True)\n",
    "            self.plots_combo_box.setCurrentIndex(self.plots_combo_box.count() - 1)\n",
    "\n",
    "            plt.clf()\n",
    "            plot_lt = plot_top5_centered_importance(self.model, cur_data, colName, True)\n",
    "            fig_lt = plot_lt.get_figure()\n",
    "            fig_lt.set_size_inches(w, h)\n",
    "            fig_lt.savefig(temp.name + f\"\\\\img1{colName}.svg\", bbox_inches=\"tight\")\n",
    "            fig_lt.savefig(\n",
    "                temp.name + f\"\\\\img1{colName}.png\", bbox_inches=\"tight\", format=\"png\"\n",
    "            )\n",
    "            plt.clf()\n",
    "\n",
    "            self.statusBar().showMessage(\"1\")\n",
    "            print(\"1\")\n",
    "\n",
    "            plot_lb = plot_ice_plot(self.model, cur_data, colName, True)\n",
    "            fig_lb = plot_lb.get_figure()\n",
    "            fig_lb.set_size_inches(w, h)\n",
    "            fig_lb.savefig(temp.name + f\"\\\\img2{colName}.svg\", bbox_inches=\"tight\")\n",
    "            fig_lb.savefig(\n",
    "                temp.name + f\"\\\\img2{colName}.png\", bbox_inches=\"tight\", format=\"png\"\n",
    "            )\n",
    "            plt.clf()\n",
    "            self.statusBar().showMessage(\"2\")\n",
    "            print(\"2\")\n",
    "\n",
    "            plot_rt = plot_top5_centered_importance(self.model, cur_data, colName)\n",
    "            fig_rt = plot_rt.get_figure().figure\n",
    "            fig_rt.set_size_inches(w, h)\n",
    "            fig_rt.savefig(temp.name + f\"\\\\img0{colName}.svg\", bbox_inches=\"tight\")\n",
    "            fig_rt.savefig(\n",
    "                temp.name + f\"\\\\img0{colName}.png\", bbox_inches=\"tight\", format=\"png\"\n",
    "            )\n",
    "            plt.clf()\n",
    "            self.statusBar().showMessage(\"3\")\n",
    "            print(\"3\")\n",
    "\n",
    "            plot_rb = plot_ice_plot(self.model, cur_data, colName)\n",
    "            fig_rb = plot_rb.get_figure().figure\n",
    "            fig_rb.set_size_inches(w, h)\n",
    "            fig_rb.savefig(temp.name + f\"\\\\img3{colName}.svg\", bbox_inches=\"tight\")\n",
    "            fig_rb.savefig(\n",
    "                temp.name + f\"\\\\img3{colName}.png\", bbox_inches=\"tight\", format=\"png\"\n",
    "            )\n",
    "            plt.clf()\n",
    "            self.statusBar().showMessage(\"5\")\n",
    "            print(\"4\")\n",
    "\n",
    "            self.show_grath(colName)\n",
    "\n",
    "        self.is_plots_in_progress = False\n",
    "\n",
    "    def __save_category_paths(\n",
    "        self, full_paths: list[[list[str]]], settings: PlotSettings\n",
    "    ) -> None:\n",
    "        name = self.__get_name_for_settings(settings)\n",
    "        for index, path in enumerate(full_paths, start=1):\n",
    "            self.__paths[f\"{name} {index}\"] = path\n",
    "\n",
    "    def __save_regular_paths(self, paths: list[str], settings: PlotSettings) -> None:\n",
    "        name = self.__get_name_for_settings(settings)\n",
    "        self.__paths[name] = paths\n",
    "\n",
    "    def __get_name_for_settings(self, settings: PlotSettings) -> str:\n",
    "        name = f\"Колонка {settings.column} от {settings.min_value} и до {settings.max_value}\"\n",
    "\n",
    "        if settings.category_column:\n",
    "            name += f\" по категории {settings.category_column}\"\n",
    "\n",
    "        return name\n",
    "\n",
    "    def __update_plot_names_in_combo_box(self):\n",
    "        plot_names = list(self.__paths.keys())\n",
    "\n",
    "        self.plots_combo_box.addItems(plot_names)\n",
    "        self.plots_combo_box.setEnabled(True)\n",
    "        self.plots_combo_box.setCurrentIndex(len(plot_names) - 1)\n",
    "\n",
    "    def show_plots(self, plots_name: str):\n",
    "        paths = self.__paths[plots_name]\n",
    "        indexes = [(0, 0), (0, 1), (1, 0), (1, 1)]\n",
    "\n",
    "        for path, position in zip(paths, indexes):\n",
    "            pixmap = PlotContainer(path)\n",
    "            pixmap.setMaximumSize(800, 600)\n",
    "            pixmap.setStyleSheet(stylesheet)\n",
    "            pixmap.setSizePolicy(\n",
    "                QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred\n",
    "            )\n",
    "            widget_to_replace = self.gridLayout.itemAtPosition(*position)\n",
    "            widget_to_replace.widget().setParent(None)\n",
    "            self.gridLayout.addWidget(pixmap, *position)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a572927-8873-4cb9-a12d-2cb1e99d821e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26cc0695-9084-4045-b400-d2f998716a47",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import shap\n",
    "\n",
    "\n",
    "\n",
    "def find_category_columns(data):\n",
    "    data_copy = data.copy()\n",
    "    categorical = []\n",
    "\n",
    "    for col in data_copy.columns:\n",
    "        if len(data_copy[col].unique()) < 10:\n",
    "            categorical.append(col)\n",
    "    return categorical\n",
    "\n",
    "\n",
    "def getMinMax(data):\n",
    "    data_copy = data.copy()\n",
    "    ar = np.array(data_copy)\n",
    "    minMax = [np.amin(ar, axis=0), np.amax(ar, axis=0)]\n",
    "    return minMax\n",
    "\n",
    "\n",
    "def top5_centered_importance(explainer, data, col_name):\n",
    "    data_copy = data.copy()\n",
    "\n",
    "    col_vals = create_variable_list(data[col_name])\n",
    "    res_vals = []\n",
    "\n",
    "    for val in col_vals:\n",
    "        new_col = len(data_copy[col_name]) * [val]\n",
    "        data_copy[col_name] = new_col\n",
    "        shap_values = explainer.shap_values(\n",
    "            data_copy.drop(\"Survived\", axis=1), y=data_copy[\"Survived\"]\n",
    "        )\n",
    "        res_vals.append(shap_values)\n",
    "\n",
    "    return res_vals, col_vals\n",
    "\n",
    "\n",
    "def top5_find_by_importance(explainer, data):\n",
    "    shap_values = explainer.shap_values(\n",
    "        data.drop(\"Survived\", axis=1), y=data[\"Survived\"]\n",
    "    )\n",
    "\n",
    "    col_names = []\n",
    "    indexes = []\n",
    "    mean_importance = list(np.mean(np.absolute(shap_values), axis=0))\n",
    "\n",
    "    max_list = sorted(mean_importance)[-5:]\n",
    "    for max_val in max_list:\n",
    "        indexes.append(mean_importance.index(max_val))\n",
    "        col_names.append(data.columns[mean_importance.index(max_val)])\n",
    "\n",
    "    return col_names, indexes\n",
    "\n",
    "\n",
    "def plot_top5_centered_importance(model, data, col_name, absolute=False):\n",
    "    plot = plt.axes()\n",
    "    plot.figure.set_size_inches(16, 8)\n",
    "\n",
    "    if absolute:\n",
    "        title = \"Центрированный график изменения абсолютной важности переменных\"\n",
    "    else:\n",
    "        title = \"Центрированный график изменения важности переменных\"\n",
    "    plot.set_title(title, fontsize=18)\n",
    "\n",
    "    explainer = shap.TreeExplainer(model)\n",
    "\n",
    "    data_copy = data.copy()\n",
    "\n",
    "    cols, indexes = top5_find_by_importance(explainer, data_copy)\n",
    "\n",
    "    res_vals, col_vals = top5_centered_importance(explainer, data_copy, col_name)\n",
    "\n",
    "    res_vals = np.array(res_vals)\n",
    "\n",
    "    for i in range(0, len(indexes)):\n",
    "        res = []\n",
    "\n",
    "        for j in range(0, len(res_vals)):\n",
    "            val = res_vals[j, :, indexes[i]]\n",
    "            if absolute:\n",
    "                val = np.absolute(val)\n",
    "            res.append(val.mean())\n",
    "\n",
    "        plot.plot(col_vals, res, color=COLORS[i], linewidth=4, label=cols[i])\n",
    "\n",
    "    plot.grid()\n",
    "    plot.set_xlabel(col_name, fontsize=16)\n",
    "    plot.set_ylabel(\"Важность переменных\", fontsize=16)\n",
    "    plot.legend()\n",
    "\n",
    "    return plot\n",
    "\n",
    "\n",
    "def ice_plot_data_y(model, data, col_name):\n",
    "    data_copy = data.copy()\n",
    "\n",
    "    col_vals = create_variable_list(data[col_name])\n",
    "    res_vals = []\n",
    "\n",
    "    for val in col_vals:\n",
    "        new_col = len(data_copy[col_name]) * [val]\n",
    "        data_copy[col_name] = new_col\n",
    "        predict = model.predict_proba(data_copy.drop(\"Survived\", axis=1))[:, 1]\n",
    "        res_vals.append(predict)\n",
    "\n",
    "    return res_vals, col_vals\n",
    "\n",
    "\n",
    "def create_variable_list(col):\n",
    "    min_val = col.min()\n",
    "    max_val = col.max()\n",
    "\n",
    "    unique = col.unique()\n",
    "\n",
    "    if len(unique) < 50:\n",
    "        col_vals = sorted(list(unique))\n",
    "    else:\n",
    "        delta = (max_val - min_val) / 100\n",
    "        col_vals = []\n",
    "        while min_val <= max_val:\n",
    "            col_vals.append(min_val)\n",
    "            min_val += delta\n",
    "    return col_vals\n",
    "\n",
    "\n",
    "def ice_plot_data_importance(explainer, data, col_name):\n",
    "    data_copy = data.copy()\n",
    "\n",
    "    col_vals = create_variable_list(data[col_name])\n",
    "    res_vals = []\n",
    "\n",
    "    for val in col_vals:\n",
    "        new_col = len(data_copy[col_name]) * [val]\n",
    "        data_copy[col_name] = new_col\n",
    "\n",
    "        shap_values = explainer.shap_values(\n",
    "            data_copy.drop(\"Survived\", axis=1), y=data_copy[\"Survived\"]\n",
    "        )\n",
    "        shap_values = np.array(shap_values)\n",
    "\n",
    "        res_vals.append(shap_values[:, list(data.columns).index(col_name)])\n",
    "\n",
    "    return res_vals, col_vals\n",
    "\n",
    "\n",
    "def plot_ice_plot(model, data, col_name, importance=False):\n",
    "    plot = plt.axes()\n",
    "\n",
    "    if importance:\n",
    "        explainer = shap.TreeExplainer(model)\n",
    "        res_vals, col_vals = ice_plot_data_importance(explainer, data, col_name)\n",
    "        y_label = f\"Важность переменной {col_name}\"\n",
    "        title = f\"с-ICE график изменения важности переменной {col_name}\"\n",
    "    else:\n",
    "        res_vals, col_vals = ice_plot_data_y(model, data, col_name)\n",
    "        y_label = \"Вероятность удачного исхода\"\n",
    "        title = f\"с-ICE график вероятности удачного исхода при изменении переменной {col_name}\"\n",
    "\n",
    "    df = pd.DataFrame(np.array(res_vals))\n",
    "    df = df.T\n",
    "    mean = df.mean()\n",
    "\n",
    "    plot.figure.set_size_inches(16, 8)\n",
    "    plot.set_title(title, fontsize=18)\n",
    "\n",
    "    for i in df.index:\n",
    "        plot.plot(col_vals, df.loc[i], color=\"black\", linewidth=0.1)\n",
    "\n",
    "    plot.plot(col_vals, mean, color=\"lime\", linewidth=6)\n",
    "\n",
    "    plot.grid()\n",
    "    plot.set_xlabel(col_name, fontsize=16)\n",
    "    plot.set_ylabel(y_label, fontsize=16)\n",
    "\n",
    "    return plot\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c14fc195-a460-48dc-9bdb-c9fde30e79a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from PyQt5 import QtGui\n",
    "\n",
    "\n",
    "stylesheet = \"\"\"\n",
    "    border-bottom-width: 1px;\n",
    "    border-bottom-style: solid;\n",
    "    border-top-width: 1px;\n",
    "    border-top-style: solid;\n",
    "    border-left-width: 1px;\n",
    "    border-left-style: solid;\n",
    "    border-right-width: 1px;\n",
    "    border-right-style: solid;\n",
    "    border-radius: 0px;\n",
    "\"\"\"\n",
    "\n",
    "COLORS = [\"brown\", \"teal\", \"blue\", \"coral\", \"limegreen\", \"pink\", \"olive\", \"navy\", \"red\"]\n",
    "FONT_SIZE = 18\n",
    "FIG_COUNT = 1\n",
    "FIG_SIZE = (1000, 1000)\n",
    "DPI = 1\n",
    "\n",
    "save_complete_message = \"Сохранение завершено\"\n",
    "\n",
    "dataset_load_error = \"Загруженный объект не является выборкой\"\n",
    "model_load_error = \"Загруженный объект не является выборкой\"\n",
    "\n",
    "data_is_awaited_message = \"Ожидается загрузка датасета и модели\"\n",
    "only_model_loaded_message = \"Загрузка модели произведена. Ожидается загрузка выборки\"\n",
    "only_dataset_loaded_message = \"Загрузка датасета произведена. Ожидается загрузка модели\"\n",
    "dataset_and_model_loaded_message = \"Датасет и модель загружены\"\n",
    "\n",
    "no_plots_message = \"Для сохранения постройте графики\"\n",
    "\n",
    "debug_file_path = \"../data/obj_v2\"\n",
    "\n",
    "\n",
    "qt_color_white = QtGui.QColor(255, 255, 255)\n",
    "qt_color_beige = QtGui.QColor(235, 204, 153)\n",
    "\n",
    "\n",
    "PLOT_WIDTH = 16\n",
    "PLOT_HEIGHT = 8\n",
    "\n",
    "TOP5_CENTERED_IMPORTANCE_TITLE = \"Центрированный график изменения важности переменных\"\n",
    "\n",
    "ICE_IMPORTANCE_Y_LABEL = \"Важность переменной {}\"\n",
    "ICE_IMPORTANCE_TITLE = \"с-ICE график изменения важности переменной {}\"\n",
    "\n",
    "ICE_PREDICTIONS_Y_LABEL = \"Вероятность удачного исхода при изменении переменной {}\"\n",
    "ICE_PREDICTIONS_TITLE = (\n",
    "    \"с-ICE график вероятности удачного исхода при изменении переменной {}\"\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "47bddea7-6a45-490b-b80f-8e3a206ca767",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n"
     ]
    },
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'app'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[3], line 5\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m DataFrame, Series\n\u001b[0;32m      4\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[1;32m----> 5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mapp\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mschemes\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmodel_explainer\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m MostImportantColumns\n\u001b[0;32m      8\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mModelExplainer\u001b[39;00m:\n\u001b[0;32m      9\u001b[0m     \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, model: XGBClassifier) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'app'"
     ]
    }
   ],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import shap\n",
    "from pandas import DataFrame, Series\n",
    "import numpy as np\n",
    "from app.schemes.model_explainer import MostImportantColumns\n",
    "\n",
    "\n",
    "class ModelExplainer:\n",
    "    def __init__(self, model: XGBClassifier) -> None:\n",
    "        self.__model = model\n",
    "        self.__explainer = shap.TreeExplainer(model)\n",
    "        self.__most_important_columns = []\n",
    "        self.__importance = {}\n",
    "        self.__predicts = {}\n",
    "\n",
    "    def get_n_most_important_columns(self, n: int) -> list[MostImportantColumns]:\n",
    "        return self.__most_important_columns[:-n]\n",
    "\n",
    "    def get_centered_importance(self):\n",
    "        return self.__importance\n",
    "\n",
    "    def get_ice_importance(self):\n",
    "        column_index = list(self.__dataset.columns).index(self.__column_to_vary)\n",
    "\n",
    "        return {\n",
    "            key: shap_values[:, column_index]\n",
    "            for key, shap_values in self.__importance.items()\n",
    "        }\n",
    "\n",
    "    def get_ice_predictions(self):\n",
    "        return self.__predicts\n",
    "\n",
    "    def calculate_for_dataset(self, dataset: DataFrame, column_to_vary: str) -> None:\n",
    "        self.__dataset = dataset\n",
    "        self.__column_to_vary = column_to_vary\n",
    "\n",
    "        self.__calculate_most_important_columns()\n",
    "        self.__calculate_importance()\n",
    "\n",
    "    def __calculate_most_important_columns(self) -> None:\n",
    "        shap_values = self.__get_shap_values(self.__dataset)\n",
    "\n",
    "        columns = []\n",
    "        mean_importance = list(np.mean(np.absolute(shap_values), axis=0))\n",
    "\n",
    "        for importance in sorted(mean_importance, reverse=True):\n",
    "            index = mean_importance.index(importance)\n",
    "            columns.append(\n",
    "                MostImportantColumns(index=index, name=self.__dataset.columns[index])\n",
    "            )\n",
    "\n",
    "        self.__most_important_columns = columns\n",
    "\n",
    "    def __calculate_importance(self) -> None:\n",
    "        dataset_copy = self.__dataset.copy()\n",
    "        column_to_vary = self.__column_to_vary\n",
    "\n",
    "        importance = {}\n",
    "        predicts = {}\n",
    "\n",
    "        dataset_len = dataset_copy[column_to_vary].count()\n",
    "\n",
    "        for value in self.__get_variables_to_vary(dataset_copy[column_to_vary]):\n",
    "            dataset_copy[column_to_vary] = dataset_len * [value]\n",
    "\n",
    "            shap_values = self.__get_shap_values(dataset_copy)\n",
    "            predict_values = self.__get_predict_values(dataset_copy)\n",
    "\n",
    "            importance[value] = shap_values\n",
    "            predicts[value] = predict_values\n",
    "\n",
    "        self.__importance = importance\n",
    "        self.__predicts = predicts\n",
    "\n",
    "    def __get_variables_to_vary(self, column: Series) -> list[float]:\n",
    "        min_val = column.min()\n",
    "        max_val = column.max()\n",
    "\n",
    "        unique = column.unique()\n",
    "\n",
    "        if len(unique) < 50:\n",
    "            col_vals = sorted(list(unique))\n",
    "        else:\n",
    "            delta = (max_val - min_val) / 100\n",
    "            col_vals = []\n",
    "\n",
    "            while min_val <= max_val:\n",
    "                col_vals.append(min_val)\n",
    "                min_val += delta\n",
    "\n",
    "        return list(col_vals)\n",
    "\n",
    "    def __get_shap_values(self, dataset: DataFrame):\n",
    "        return np.array(\n",
    "            self.__explainer.shap_values(\n",
    "                dataset.drop(\"Survived\", axis=1), y=dataset[\"Survived\"]\n",
    "            )\n",
    "        )\n",
    "\n",
    "    def __get_predict_values(self, dataset: DataFrame):\n",
    "        return np.array(\n",
    "            self.__model.predict_proba(dataset.drop(\"Survived\", axis=1))[:, 1]\n",
    "        )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5e98ce5-ec5b-4f31-9fbc-40c66ac83a86",
   "metadata": {},
   "outputs": [],
   "source": [
    "from .."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
